{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "W3_Homework.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CIS-522/course-content/blob/main/tutorials/W03_MLPs/W3_Homework.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_li9itVYjjpg"
      },
      "source": [
        "# CIS-522 Week 3 Homework\r\n",
        "\r\n",
        "\r\n",
        "**Instructor:** Konrad Kording\r\n",
        "\r\n",
        "**Content Creators:** Arash Ash, Jordan Lei"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pOoyXdCFijaM",
        "cellView": "form"
      },
      "source": [
        "#@markdown What is your Pennkey and pod? (text, not numbers, e.g. bfranklin)\r\n",
        "my_pennkey = '' #@param {type:\"string\"}\r\n",
        "my_pod = 'Select' #@param ['Select', 'euclidean-wombat', 'sublime-newt', 'buoyant-unicorn', 'lackadaisical-manatee','indelible-stingray','superfluous-lyrebird','discreet-reindeer','quizzical-goldfish','ubiquitous-cheetah','nonchalant-crocodile','fashionable-lemur','spiffy-eagle','electric-emu','quotidian-lion','astute-jellyfish', 'quantum-herring']\r\n",
        "\r\n",
        "# start timing\r\n",
        "import time\r\n",
        "try:t0;\r\n",
        "except NameError: t0 = time.time()\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XWOc48yQrvUI"
      },
      "source": [
        "---\r\n",
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mLNTKY8sEUj"
      },
      "source": [
        "# imports\r\n",
        "import os\r\n",
        "import csv\r\n",
        "import json\r\n",
        "import random\r\n",
        "import pathlib\r\n",
        "\r\n",
        "import torch\r\n",
        "import numpy as np\r\n",
        "import pandas as pd\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "\r\n",
        "import torch.nn as nn\r\n",
        "import torch.optim as optim\r\n",
        "import torch.nn.functional as F\r\n",
        "from torch.utils.data import DataLoader, TensorDataset\r\n",
        "from IPython.display import HTML, display\r\n",
        "\r\n",
        "dev = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\r\n",
        "dev, torch.get_num_threads()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KCEnJU8StUzq"
      },
      "source": [
        "# @title Install Kaggle API (run me)\r\n",
        "# %%capture\r\n",
        "!pip install --upgrade --force-reinstall --no-deps kaggle==1.5.10\r\n",
        "!kaggle -v"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hFpjwPq78MdR",
        "cellView": "form"
      },
      "source": [
        "# @title Helper functions\r\n",
        "def imshow(img):\r\n",
        "    img = img / 2 + 0.5     # unnormalize\r\n",
        "    npimg = img.numpy()\r\n",
        "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\r\n",
        "    plt.axis(False)\r\n",
        "    plt.show()\r\n",
        "\r\n",
        "def progress(epoch, loss, epochs=100):\r\n",
        "    return HTML(\"\"\"\r\n",
        "        <label for=\"file\">Training loss: {loss}</label>\r\n",
        "        <progress\r\n",
        "            value='{epoch}'\r\n",
        "            max='{epochs}',\r\n",
        "            style='width: 100%'\r\n",
        "        >\r\n",
        "            {epoch}\r\n",
        "        </progress>\r\n",
        "    \"\"\".format(loss=loss, epoch=epoch, epochs=epochs))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QxpHO4kBkU3e"
      },
      "source": [
        "# Part 1. Kaggle Competition\r\n",
        "To use the Kaggle API, sign up for a Kaggle account at https://www.kaggle.com. Then go to the 'Account' tab of your user profile (`https://www.kaggle.com/<username>/account`) and select 'Create API Token'. This will trigger the download of `kaggle.json`, a file containing your API credentials. Place this file in Colab Files on the left menu.\r\n",
        "\r\n",
        "Then join the competition and accept the rules at https://www.kaggle.com/c/permuted-animal-faces/rules.\r\n",
        "Now we can handle the submisions here using the Kaggle API:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tafGG7zF_ipg"
      },
      "source": [
        "![image.png]()"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfDMhvNwzTxx"
      },
      "source": [
        "# registering your API key\r\n",
        "!mkdir ~/.kaggle\r\n",
        "!cp kaggle.json ~/.kaggle/\r\n",
        "!chmod 600 ~/.kaggle/kaggle.json"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6D0MvzgrzioP"
      },
      "source": [
        "!kaggle competitions download -c permuted-animal-faces\r\n",
        "!unzip permuted-animal-faces.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jt270XcTDaRH"
      },
      "source": [
        "Now it's your turn to train a competitive model using what you learned about MLPs and show it off to the rest of the world in the Kaggle leaderboard.\r\n",
        "\r\n",
        "Here we provided you the same code as in the tutorial and the submission code, which you can run and get a baseline result. But we encourage you to get creative and combine your previous ML know-how since nothing is off the table in this competition."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LFSQh1demt0n"
      },
      "source": [
        "class Net(nn.Module):\r\n",
        "    def __init__(self, actv, num_inputs, hidden_units, num_outputs):\r\n",
        "        super(Net, self).__init__()\r\n",
        "\r\n",
        "        exec('self.actv = nn.%s'%actv)   # [TO-DO]\r\n",
        "\r\n",
        "        self.layers = nn.ModuleList()\r\n",
        "        for i in range(len(hidden_units)):\r\n",
        "          next_num_inputs = hidden_units[i] \r\n",
        "          self.layers += [nn.Linear(num_inputs, next_num_inputs)]   # [TO-DO]\r\n",
        "          num_inputs = next_num_inputs\r\n",
        "\r\n",
        "        self.out = nn.Linear(num_inputs, num_outputs)\r\n",
        "\r\n",
        "    def forward(self, x):\r\n",
        "        # flattening\r\n",
        "        x = x.view(x.shape[0], -1)   # [TO-DO]\r\n",
        "\r\n",
        "        for layer in self.layers:\r\n",
        "          x = self.actv(layer(x))  # [TO-DO]\r\n",
        "        x = self.out(x) # [TO-DO]\r\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xb23NZ3l4BUB"
      },
      "source": [
        "df = pd.read_csv('train.csv')\r\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XB9gCLi65txD"
      },
      "source": [
        "data = np.array(df)\r\n",
        "X_train = torch.tensor(data[:, 1:]).float()/255\r\n",
        "y_train = torch.tensor(data[:, 0]).long()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "laqU5kuxwOnD"
      },
      "source": [
        "batch_size = 128\r\n",
        "train_data = TensorDataset(X_train, y_train)\r\n",
        "train_loader = DataLoader(train_data, batch_size=batch_size, drop_last=True,\r\n",
        "                        shuffle=True, num_workers=10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6DUVNbJiwSfj"
      },
      "source": [
        "def train_test_classification(net, criterion, optimizer,\r\n",
        "                              train_loader,\r\n",
        "                              num_epochs=1, verbose=True,\r\n",
        "                              training_plot=True):\r\n",
        "  if verbose:\r\n",
        "    progress_bar = display(progress(0, 0, num_epochs), display_id=True)\r\n",
        "\r\n",
        "  net.train()\r\n",
        "  training_losses = []\r\n",
        "  for epoch in range(num_epochs):  # loop over the dataset multiple times\r\n",
        "      running_loss = 0.0\r\n",
        "      for i, data in enumerate(train_loader, 0):\r\n",
        "          # get the inputs; data is a list of [inputs, labels]\r\n",
        "          inputs, labels = data\r\n",
        "          inputs = inputs.to(dev).float()\r\n",
        "          labels = labels.to(dev).long()\r\n",
        "\r\n",
        "          # zero the parameter gradients\r\n",
        "          optimizer.zero_grad()\r\n",
        "\r\n",
        "          # forward + backward + optimize\r\n",
        "          outputs = net(inputs)\r\n",
        "\r\n",
        "          loss = criterion(outputs, labels)\r\n",
        "          loss.backward()\r\n",
        "          optimizer.step()\r\n",
        "\r\n",
        "          # print statistics\r\n",
        "          if verbose:\r\n",
        "            training_losses += [loss.item()]\r\n",
        "            running_loss += loss.item()\r\n",
        "            if i % 10 == 9:    # update every 10 mini-batches\r\n",
        "                progress_bar.update(progress(epoch+1, running_loss / 10, num_epochs))\r\n",
        "                running_loss = 0.0\r\n",
        "\r\n",
        "  net.eval()\r\n",
        "  def test(data_loader):\r\n",
        "    correct = 0\r\n",
        "    total = 0\r\n",
        "    for data in data_loader:\r\n",
        "        inputs, labels = data\r\n",
        "        inputs = inputs.to(dev).float()\r\n",
        "        labels = labels.to(dev).long()\r\n",
        "\r\n",
        "        outputs = net(inputs)\r\n",
        "        _, predicted = torch.max(outputs, 1)\r\n",
        "        total += labels.size(0)\r\n",
        "        correct += (predicted == labels).sum().item()\r\n",
        "\r\n",
        "    acc = 100 * correct / total\r\n",
        "    return total, acc\r\n",
        "\r\n",
        "  train_total, train_acc = test(train_loader)\r\n",
        "\r\n",
        "  if verbose:\r\n",
        "    print('Accuracy on the %d training samples: %0.2f %%' % (train_total, train_acc))\r\n",
        "\r\n",
        "  if training_plot:\r\n",
        "    plt.plot(training_losses)\r\n",
        "    plt.xlabel('Batch')\r\n",
        "    plt.ylabel('Training loss')\r\n",
        "    plt.show()\r\n",
        "  \r\n",
        "  return train_acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJd1-YuAwau6"
      },
      "source": [
        "net = Net('ReLU()', 3*32*32, [128, 32], 3).to(dev) \r\n",
        "criterion = nn.MultiMarginLoss(margin=1.0)\r\n",
        "optimizer = optim.Adam(net.parameters(), lr=3e-4)\r\n",
        "train_acc = train_test_classification(net, criterion, optimizer,\r\n",
        "                                      train_loader,\r\n",
        "                                      num_epochs=50)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9Vzhqtr7c6Y"
      },
      "source": [
        "# Predict test set\r\n",
        "df = pd.read_csv('test.csv')\r\n",
        "data = np.array(df)\r\n",
        "X_test = torch.tensor(data).float()/255\r\n",
        "\r\n",
        "net.eval()\r\n",
        "y_pred = net(X_test.to(dev))\r\n",
        "labels_pred = torch.argmax(y_pred, dim=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAOeoZH48OuJ"
      },
      "source": [
        "# create the submission file\r\n",
        "header = ['ImageId', 'Label']\r\n",
        "with open('submission.csv', 'w', newline='') as file:\r\n",
        "    writer = csv.writer(file)\r\n",
        "    writer.writerow(header)\r\n",
        "    \r\n",
        "    for i in range(len(labels_pred)):\r\n",
        "      writer.writerow([i+1] + [labels_pred[i].item()])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XEojRbYH9adp"
      },
      "source": [
        "# submit\r\n",
        "!kaggle competitions submit permuted-animal-faces -f submission.csv -m \"Result with same model as in the tutorial\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p_gDgatsCKJL"
      },
      "source": [
        "Note that you can submit maximum 20 times a day. Participants will need to wait until the next UTC day after submitting the maximum number of daily submissions.\r\n",
        "\r\n",
        "And finally you can hand select 2 submissions among your previous ones to be used for the final ranking."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_GEV3MSbDQX3"
      },
      "source": [
        "# Part 2. Deep Learning, Deeper Thinking\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2vh4iLLj0qN0"
      },
      "source": [
        "## 2.1 Our Data-Processing Wish List\n",
        "\n",
        "By now you should be pretty familiar with classifying animal faces using multi-layer perceptrons. MLPs make very few assumptions about the underlying structure of the data, which make them very flexible in approximating a wide variety of functions. However, sometimes it can be helpful to constrain the set of possible functions we want to approximate by introducing some \"rules\" that these functions need to follow. We call these rules **inductive biases**. \n",
        "\n",
        "For example, one inductive bias might be the rule that \"A cat is a cat, no matter where it is placed in the image. An upper-left-corner cat is just as much of a cat as a lower-right-corner cat.\" That describes an inductive bias called **translational invariance**, which basically says that the class of an object is invariant to translation (it doesn't matter where it is, as long as it's there). If we could have a model that preserves translational invariance, chances are it would be helpful in finding us good models for image processing (more on this later). \n",
        "\n",
        "**Now that you're familiar with the concept of inductive biases, please do the following:**\n",
        "* List one inductive bias other than translational invariance that would be good to have for image processing\n",
        "* Suppose you're trying to develop a model to assign credit scores to people based on their personal data. What kind of inductive biases or data guarantees would you want to impose on this system?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7DfgGaluts9P"
      },
      "source": [
        "#Delete the starter text, enter your response, and run the cell.\n",
        "q_2_1 = '''\n",
        "Your Response to Q 2.1 Here\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RxuEtqNmxqkT"
      },
      "source": [
        "## 2.2 The Ethics of Facial Recognition\n",
        "In part 1, we just had you create a network that was good at classifying faces of animals. What happens when this technologies are used on humans? What happens when these technologies are used by law enforcement? In this section, we'll explore these implications. So play your favorite Lo-Fi Beats to Study/Relax To and let's get started!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RO8qZ_IDxL8g"
      },
      "source": [
        "### 2.2.1 **America Under Watch**\n",
        "As data scientists, we often complain that we don't have enough data. Sometimes lack of good data isn't the problem. What happens if we collect data *everywhere*? In many countries around the world, facial recognition is used by law enforcement to conduct wide-scale law enforcement. Thankfully that's not the case in the United States. Right? \n",
        "\n",
        "*... Right?* \n",
        "\n",
        "Read the following:\n",
        "\n",
        "* https://www.americaunderwatch.com/\n",
        "\n",
        "**In 200-300 words, please answer the following questions.** \n",
        "* In America Under Watch, the Georgetown Law Center proposes a moratorium (temporary freeze) on the police use of facial recognition. Do you agree or disagree with this choice, and why? \n",
        "* The article mentions the fact that *facial surveillance is disproportionately likely to impact communities of color and communities with lower socio-economic status (SES)*. Think about what this means about what amount + what kinds of data law enforcement collect on people of color. How might this perpetuate cycles of incarceration for people of color in the U.S.?\n",
        "* If facial recognition were to be put in place in a highly secure location, like the White House, what kind of inductive biases would you want to ensure that such a system would have? What guarantees might you want to have about the data that is used to train the network, the data that is collected, and what is done with the data afterward? \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i7Bz8XqX26yt"
      },
      "source": [
        "#Delete the starter text, enter your response, and run the cell.\n",
        "q_2_2_1 = '''\n",
        "Your Response to Q 2.2.1 Here (200 - 300 words)\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b5qeXoWo3Bb6"
      },
      "source": [
        "### 2.2.2 **Grab Some Popcorn**\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "wZ8751Th3Vh4"
      },
      "source": [
        "#@markdown That's enough reading for today, don't you think? Grab some popcorn, run this cell, and watch this video\n",
        "from IPython.display import YouTubeVideo\n",
        "video = YouTubeVideo(id=\"jZjmlJPJgug\", width=854, height=480, fs=1)\n",
        "print(\"Video available at https://youtube.com/watch?v=\" + video.id)\n",
        "\n",
        "video"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T79vzE1K4Ho7"
      },
      "source": [
        "**In 200-300 words, please answer the following questions**\n",
        "* In the segment, John Oliver mentions that Clearview AI uses \"publicly available data\", which includes public profile information on Facebook, LinkedIn, Twitter, and Instagram. Clearview argues that this information has all been willingly made public. Do you agree? Why or why not?\n",
        "* Some prominent data scientists claim that \"data/algorithms don't have bias, people have bias\". Do you agree with this sentiment? Why or why not?\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqFIndfb4ubV"
      },
      "source": [
        "#Delete the starter text, enter your response, and run the cell.\n",
        "q_2_2_2 = '''\n",
        "Your Response to Q 2.2.2 Here (200 - 300 words)\n",
        "\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lxsQXMFc8v91"
      },
      "source": [
        "# Submit to Airtable\n",
        "**Don't forget to contribute to the conversation in your pod slack channel.** You can do so by copying and pasting some or all of the answers to the above questions, or by commenting and responding to other people's posts.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AlH_Ub579KEJ",
        "cellView": "form"
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "from IPython.display import IFrame\n",
        "#@markdown #Run Cell to Show Airtable Form\n",
        "#@markdown ##**Confirm your answers and then click \"Submit\"**\n",
        "\n",
        "def prefill_form(src, fields: dict):\n",
        "  '''\n",
        "  src: the original src url to embed the form\n",
        "  fields: a dictionary of field:value pairs,\n",
        "  e.g. {\"pennkey\": my_pennkey, \"location\": my_location}\n",
        "  '''\n",
        "  prefills = \"&\".join([\"prefill_%s=%s\"%(key, fields[key]) for key in fields])\n",
        "  src = src + prefills\n",
        "  src = \"+\".join(src.split(\" \"))\n",
        "  return src\n",
        "\n",
        "\n",
        "#autofill time if it is not present\n",
        "try: t0;\n",
        "except NameError: t0 = time.time()\n",
        "try: t1;\n",
        "except NameError: t1 = time.time()\n",
        "\n",
        "#autofill fields if they are not present\n",
        "#a missing pennkey and pod will result in an Airtable warning\n",
        "#which is easily fixed user-side.\n",
        "try: my_pennkey;\n",
        "except NameError: my_pennkey = \"\"\n",
        "try: my_pod;\n",
        "except NameError: my_pod = \"Select\"\n",
        "try: q_2_1;\n",
        "except NameError: q_2_1 = \"\"\n",
        "try: q_2_2_1;\n",
        "except NameError: q_2_2_1 = \"\"\n",
        "try: q_2_2_2;\n",
        "except NameError: q_2_2_2 = \"\"\n",
        "\n",
        "\n",
        "times = np.array([t1])-t0\n",
        "\n",
        "fields = {\"pennkey\": my_pennkey,\n",
        "          \"pod\": my_pod,\n",
        "          \"q_2_1\": q_2_1,\n",
        "          \"q_2_2_1\": q_2_2_1,\n",
        "          \"q_2_2_2\": q_2_2_2,\n",
        "          \"cumulative_times\": times}\n",
        "\n",
        "src = \"https://airtable.com/embed/shrzuKoahe4ZQKvND?\"\n",
        "\n",
        "#now instead of the original source url, we do: src = prefill_form(src, fields)\n",
        "display(IFrame(src = prefill_form(src, fields), width = 800, height = 400))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}